{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: plotnine in /data/paras/miniconda3/lib/python3.8/site-packages (0.7.1)\n",
      "Requirement already satisfied: numpy in /data/paras/miniconda3/lib/python3.8/site-packages (1.19.2)\n",
      "Requirement already satisfied: pandas in /data/paras/miniconda3/lib/python3.8/site-packages (1.1.3)\n",
      "Requirement already satisfied: swifter in /data/paras/miniconda3/lib/python3.8/site-packages (1.0.7)\n",
      "Requirement already satisfied: tqdm in /data/paras/miniconda3/lib/python3.8/site-packages (4.50.2)\n",
      "Requirement already satisfied: sentencepiece in /data/paras/miniconda3/lib/python3.8/site-packages (0.1.94)\n",
      "Requirement already satisfied: matplotlib>=3.1.1 in /data/paras/miniconda3/lib/python3.8/site-packages (from plotnine) (3.3.2)\n",
      "Requirement already satisfied: patsy>=0.5.1 in /data/paras/miniconda3/lib/python3.8/site-packages (from plotnine) (0.5.1)\n",
      "Requirement already satisfied: statsmodels>=0.11.1 in /data/paras/miniconda3/lib/python3.8/site-packages (from plotnine) (0.12.1)\n",
      "Requirement already satisfied: descartes>=1.1.0 in /data/paras/miniconda3/lib/python3.8/site-packages (from plotnine) (1.1.0)\n",
      "Requirement already satisfied: scipy>=1.2.0 in /data/paras/miniconda3/lib/python3.8/site-packages (from plotnine) (1.5.2)\n",
      "Requirement already satisfied: mizani>=0.7.1 in /data/paras/miniconda3/lib/python3.8/site-packages (from plotnine) (0.7.2)\n",
      "Requirement already satisfied: pytz>=2017.2 in /data/paras/miniconda3/lib/python3.8/site-packages (from pandas) (2020.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /data/paras/miniconda3/lib/python3.8/site-packages (from pandas) (2.8.1)\n",
      "Requirement already satisfied: ipywidgets>=7.0.0cloudpickle>=0.2.2 in /data/paras/miniconda3/lib/python3.8/site-packages (from swifter) (7.5.1)\n",
      "Requirement already satisfied: bleach>=3.1.1 in /data/paras/miniconda3/lib/python3.8/site-packages (from swifter) (3.2.1)\n",
      "Requirement already satisfied: parso>0.4.0 in /data/paras/miniconda3/lib/python3.8/site-packages (from swifter) (0.7.0)\n",
      "Requirement already satisfied: modin[ray]>=0.8.1.1 in /data/paras/miniconda3/lib/python3.8/site-packages (from swifter) (0.8.2)\n",
      "Requirement already satisfied: dask[dataframe]>=2.10.0 in /data/paras/miniconda3/lib/python3.8/site-packages (from swifter) (2.30.0)\n",
      "Requirement already satisfied: psutil>=5.6.6 in /data/paras/miniconda3/lib/python3.8/site-packages (from swifter) (5.7.3)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /data/paras/miniconda3/lib/python3.8/site-packages (from matplotlib>=3.1.1->plotnine) (1.3.0)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /data/paras/miniconda3/lib/python3.8/site-packages (from matplotlib>=3.1.1->plotnine) (8.0.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /data/paras/miniconda3/lib/python3.8/site-packages (from matplotlib>=3.1.1->plotnine) (0.10.0)\n",
      "Requirement already satisfied: certifi>=2020.06.20 in /data/paras/miniconda3/lib/python3.8/site-packages (from matplotlib>=3.1.1->plotnine) (2020.6.20)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.3 in /data/paras/miniconda3/lib/python3.8/site-packages (from matplotlib>=3.1.1->plotnine) (2.4.7)\n",
      "Requirement already satisfied: six in /data/paras/miniconda3/lib/python3.8/site-packages (from patsy>=0.5.1->plotnine) (1.15.0)\n",
      "Requirement already satisfied: palettable in /data/paras/miniconda3/lib/python3.8/site-packages (from mizani>=0.7.1->plotnine) (3.3.0)\n",
      "Requirement already satisfied: widgetsnbextension~=3.5.0 in /data/paras/miniconda3/lib/python3.8/site-packages (from ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (3.5.1)\n",
      "Requirement already satisfied: traitlets>=4.3.1 in /data/paras/miniconda3/lib/python3.8/site-packages (from ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (5.0.5)\n",
      "Requirement already satisfied: ipykernel>=4.5.1 in /data/paras/miniconda3/lib/python3.8/site-packages (from ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (5.3.4)\n",
      "Requirement already satisfied: nbformat>=4.2.0 in /data/paras/miniconda3/lib/python3.8/site-packages (from ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (5.0.8)\n",
      "Requirement already satisfied: ipython>=4.0.0; python_version >= \"3.3\" in /data/paras/miniconda3/lib/python3.8/site-packages (from ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (7.19.0)\n",
      "Requirement already satisfied: packaging in /data/paras/miniconda3/lib/python3.8/site-packages (from bleach>=3.1.1->swifter) (20.4)\n",
      "Requirement already satisfied: webencodings in /data/paras/miniconda3/lib/python3.8/site-packages (from bleach>=3.1.1->swifter) (0.5.1)\n",
      "Requirement already satisfied: ray>=1.0.0; extra == \"ray\" in /data/paras/miniconda3/lib/python3.8/site-packages (from modin[ray]>=0.8.1.1->swifter) (1.0.1)\n",
      "Requirement already satisfied: pyarrow==1.0; extra == \"ray\" in /data/paras/miniconda3/lib/python3.8/site-packages (from modin[ray]>=0.8.1.1->swifter) (1.0.0)\n",
      "Requirement already satisfied: pyyaml in /data/paras/miniconda3/lib/python3.8/site-packages (from dask[dataframe]>=2.10.0->swifter) (5.3.1)\n",
      "Requirement already satisfied: toolz>=0.8.2; extra == \"dataframe\" in /data/paras/miniconda3/lib/python3.8/site-packages (from dask[dataframe]>=2.10.0->swifter) (0.11.1)\n",
      "Requirement already satisfied: partd>=0.3.10; extra == \"dataframe\" in /data/paras/miniconda3/lib/python3.8/site-packages (from dask[dataframe]>=2.10.0->swifter) (1.1.0)\n",
      "Requirement already satisfied: fsspec>=0.6.0; extra == \"dataframe\" in /data/paras/miniconda3/lib/python3.8/site-packages (from dask[dataframe]>=2.10.0->swifter) (0.8.4)\n",
      "Requirement already satisfied: notebook>=4.4.1 in /data/paras/miniconda3/lib/python3.8/site-packages (from widgetsnbextension~=3.5.0->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (6.1.4)\n",
      "Requirement already satisfied: ipython-genutils in /data/paras/miniconda3/lib/python3.8/site-packages (from traitlets>=4.3.1->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (0.2.0)\n",
      "Requirement already satisfied: jupyter-client in /data/paras/miniconda3/lib/python3.8/site-packages (from ipykernel>=4.5.1->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (6.1.7)\n",
      "Requirement already satisfied: tornado>=4.2 in /data/paras/miniconda3/lib/python3.8/site-packages (from ipykernel>=4.5.1->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (6.0.4)\n",
      "Requirement already satisfied: jupyter-core in /data/paras/miniconda3/lib/python3.8/site-packages (from nbformat>=4.2.0->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (4.6.3)\n",
      "Requirement already satisfied: jsonschema!=2.5.0,>=2.4 in /data/paras/miniconda3/lib/python3.8/site-packages (from nbformat>=4.2.0->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (3.2.0)\n",
      "Requirement already satisfied: jedi>=0.10 in /data/paras/miniconda3/lib/python3.8/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (0.17.2)\n",
      "Requirement already satisfied: pexpect>4.3; sys_platform != \"win32\" in /data/paras/miniconda3/lib/python3.8/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (4.8.0)\n",
      "Requirement already satisfied: setuptools>=18.5 in /data/paras/miniconda3/lib/python3.8/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (50.3.0.post20201103)\n",
      "Requirement already satisfied: pygments in /data/paras/miniconda3/lib/python3.8/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (2.7.2)\n",
      "Requirement already satisfied: pickleshare in /data/paras/miniconda3/lib/python3.8/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (0.7.5)\n",
      "Requirement already satisfied: backcall in /data/paras/miniconda3/lib/python3.8/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (0.2.0)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /data/paras/miniconda3/lib/python3.8/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (3.0.8)\n",
      "Requirement already satisfied: decorator in /data/paras/miniconda3/lib/python3.8/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (4.4.2)\n",
      "Requirement already satisfied: aiohttp-cors in /data/paras/miniconda3/lib/python3.8/site-packages (from ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (0.7.0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: py-spy>=0.2.0 in /data/paras/miniconda3/lib/python3.8/site-packages (from ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (0.3.3)\n",
      "Requirement already satisfied: grpcio>=1.28.1 in /data/paras/miniconda3/lib/python3.8/site-packages (from ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (1.31.0)\n",
      "Requirement already satisfied: filelock in /data/paras/miniconda3/lib/python3.8/site-packages (from ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (3.0.12)\n",
      "Requirement already satisfied: colorama in /data/paras/miniconda3/lib/python3.8/site-packages (from ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (0.4.4)\n",
      "Requirement already satisfied: prometheus-client>=0.7.1 in /data/paras/miniconda3/lib/python3.8/site-packages (from ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (0.8.0)\n",
      "Requirement already satisfied: protobuf>=3.8.0 in /data/paras/miniconda3/lib/python3.8/site-packages (from ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (3.13.0)\n",
      "Requirement already satisfied: redis<3.5.0,>=3.3.2 in /data/paras/miniconda3/lib/python3.8/site-packages (from ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (3.4.1)\n",
      "Requirement already satisfied: aiohttp in /data/paras/miniconda3/lib/python3.8/site-packages (from ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (3.6.3)\n",
      "Requirement already satisfied: colorful in /data/paras/miniconda3/lib/python3.8/site-packages (from ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (0.5.4)\n",
      "Requirement already satisfied: google in /data/paras/miniconda3/lib/python3.8/site-packages (from ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (3.0.0)\n",
      "Requirement already satisfied: requests in /data/paras/miniconda3/lib/python3.8/site-packages (from ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (2.24.0)\n",
      "Requirement already satisfied: aioredis in /data/paras/miniconda3/lib/python3.8/site-packages (from ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (1.3.1)\n",
      "Requirement already satisfied: gpustat in /data/paras/miniconda3/lib/python3.8/site-packages (from ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (0.6.0)\n",
      "Requirement already satisfied: opencensus in /data/paras/miniconda3/lib/python3.8/site-packages (from ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (0.7.11)\n",
      "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /data/paras/miniconda3/lib/python3.8/site-packages (from ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (1.0.0)\n",
      "Requirement already satisfied: click>=7.0 in /data/paras/miniconda3/lib/python3.8/site-packages (from ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (7.1.2)\n",
      "Requirement already satisfied: locket in /data/paras/miniconda3/lib/python3.8/site-packages (from partd>=0.3.10; extra == \"dataframe\"->dask[dataframe]>=2.10.0->swifter) (0.2.0)\n",
      "Requirement already satisfied: Send2Trash in /data/paras/miniconda3/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (1.5.0)\n",
      "Requirement already satisfied: pyzmq>=17 in /data/paras/miniconda3/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (19.0.2)\n",
      "Requirement already satisfied: terminado>=0.8.3 in /data/paras/miniconda3/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (0.9.1)\n",
      "Requirement already satisfied: jinja2 in /data/paras/miniconda3/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (2.11.2)\n",
      "Requirement already satisfied: nbconvert in /data/paras/miniconda3/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (6.0.7)\n",
      "Requirement already satisfied: argon2-cffi in /data/paras/miniconda3/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (20.1.0)\n",
      "Requirement already satisfied: attrs>=17.4.0 in /data/paras/miniconda3/lib/python3.8/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.2.0->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (20.2.0)\n",
      "Requirement already satisfied: pyrsistent>=0.14.0 in /data/paras/miniconda3/lib/python3.8/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.2.0->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (0.17.3)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /data/paras/miniconda3/lib/python3.8/site-packages (from pexpect>4.3; sys_platform != \"win32\"->ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (0.6.0)\n",
      "Requirement already satisfied: wcwidth in /data/paras/miniconda3/lib/python3.8/site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (0.2.5)\n",
      "Requirement already satisfied: chardet<4.0,>=2.0 in /data/paras/miniconda3/lib/python3.8/site-packages (from aiohttp->ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (3.0.4)\n",
      "Requirement already satisfied: yarl<1.6.0,>=1.0 in /data/paras/miniconda3/lib/python3.8/site-packages (from aiohttp->ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (1.5.1)\n",
      "Requirement already satisfied: multidict<5.0,>=4.5 in /data/paras/miniconda3/lib/python3.8/site-packages (from aiohttp->ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (4.7.6)\n",
      "Requirement already satisfied: async-timeout<4.0,>=3.0 in /data/paras/miniconda3/lib/python3.8/site-packages (from aiohttp->ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (3.0.1)\n",
      "Requirement already satisfied: beautifulsoup4 in /data/paras/miniconda3/lib/python3.8/site-packages (from google->ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (4.9.3)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /data/paras/miniconda3/lib/python3.8/site-packages (from requests->ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (1.25.11)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /data/paras/miniconda3/lib/python3.8/site-packages (from requests->ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (2.10)\n",
      "Requirement already satisfied: hiredis in /data/paras/miniconda3/lib/python3.8/site-packages (from aioredis->ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (1.1.0)\n",
      "Requirement already satisfied: blessings>=1.6 in /data/paras/miniconda3/lib/python3.8/site-packages (from gpustat->ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (1.7)\n",
      "Requirement already satisfied: nvidia-ml-py3>=7.352.0 in /data/paras/miniconda3/lib/python3.8/site-packages (from gpustat->ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (7.352.0)\n",
      "Requirement already satisfied: opencensus-context==0.1.2 in /data/paras/miniconda3/lib/python3.8/site-packages (from opencensus->ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (0.1.2)\n",
      "Requirement already satisfied: google-api-core<2.0.0,>=1.0.0 in /data/paras/miniconda3/lib/python3.8/site-packages (from opencensus->ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (1.23.0)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in /data/paras/miniconda3/lib/python3.8/site-packages (from jinja2->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (1.1.1)\n",
      "Requirement already satisfied: pandocfilters>=1.4.1 in /data/paras/miniconda3/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (1.4.2)\n",
      "Requirement already satisfied: jupyterlab-pygments in /data/paras/miniconda3/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (0.1.2)\n",
      "Requirement already satisfied: testpath in /data/paras/miniconda3/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (0.4.4)\n",
      "Requirement already satisfied: entrypoints>=0.2.2 in /data/paras/miniconda3/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (0.3)\n",
      "Requirement already satisfied: mistune<2,>=0.8.1 in /data/paras/miniconda3/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (0.8.4)\n",
      "Requirement already satisfied: defusedxml in /data/paras/miniconda3/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (0.6.0)\n",
      "Requirement already satisfied: nbclient<0.6.0,>=0.5.0 in /data/paras/miniconda3/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (0.5.1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: cffi>=1.0.0 in /data/paras/miniconda3/lib/python3.8/site-packages (from argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (1.14.3)\n",
      "Requirement already satisfied: soupsieve>1.2; python_version >= \"3.0\" in /data/paras/miniconda3/lib/python3.8/site-packages (from beautifulsoup4->google->ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (2.0.1)\n",
      "Requirement already satisfied: google-auth<2.0dev,>=1.21.1 in /data/paras/miniconda3/lib/python3.8/site-packages (from google-api-core<2.0.0,>=1.0.0->opencensus->ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (1.23.0)\n",
      "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.6.0 in /data/paras/miniconda3/lib/python3.8/site-packages (from google-api-core<2.0.0,>=1.0.0->opencensus->ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (1.52.0)\n",
      "Requirement already satisfied: nest-asyncio in /data/paras/miniconda3/lib/python3.8/site-packages (from nbclient<0.6.0,>=0.5.0->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (1.4.1)\n",
      "Requirement already satisfied: async-generator in /data/paras/miniconda3/lib/python3.8/site-packages (from nbclient<0.6.0,>=0.5.0->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (1.10)\n",
      "Requirement already satisfied: pycparser in /data/paras/miniconda3/lib/python3.8/site-packages (from cffi>=1.0.0->argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0cloudpickle>=0.2.2->swifter) (2.20)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /data/paras/miniconda3/lib/python3.8/site-packages (from google-auth<2.0dev,>=1.21.1->google-api-core<2.0.0,>=1.0.0->opencensus->ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (4.1.1)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /data/paras/miniconda3/lib/python3.8/site-packages (from google-auth<2.0dev,>=1.21.1->google-api-core<2.0.0,>=1.0.0->opencensus->ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (0.2.8)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4; python_version >= \"3.5\" in /data/paras/miniconda3/lib/python3.8/site-packages (from google-auth<2.0dev,>=1.21.1->google-api-core<2.0.0,>=1.0.0->opencensus->ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (4.6)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /data/paras/miniconda3/lib/python3.8/site-packages (from pyasn1-modules>=0.2.1->google-auth<2.0dev,>=1.21.1->google-api-core<2.0.0,>=1.0.0->opencensus->ray>=1.0.0; extra == \"ray\"->modin[ray]>=0.8.1.1->swifter) (0.4.8)\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "/work/paras/code/contracode\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%pip install plotnine numpy pandas swifter tqdm sentencepiece\n",
    "%cd /work/paras/code/contracode\n",
    "%pwd\n",
    "%matplotlib inline\n",
    "!mkdir -p data/plots\n",
    "import jsonlines\n",
    "import sentencepiece as spm\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import os\n",
    "import plotnine as p9\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas()\n",
    "\n",
    "sp = spm.SentencePieceProcessor()\n",
    "sp.Load('data/codesearchnet_javascript/csnjs_8k_9995p_unigram_url.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "camel_case_re = re.compile(r\".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)\")\n",
    "_fix_function_crop_regexes = [\n",
    "    re.compile(r + r\"(\\s+|\\()\") for r in [r\"\\A^unction\", r\"\\A^nction\", r\"\\A^ction\", r\"\\A^tion\", r\"\\A^ion\", r\"\\A^on\", r\"\\A^n\"]\n",
    "]\n",
    "_valid_identifier_regex = re.compile(r\"^[a-zA-Z_$][0-9a-zA-Z_$]*$\")\n",
    "_url_regex = re.compile(r\"https?://\\S+\\b\")\n",
    "\n",
    "\n",
    "def camel_case_split(identifier):\n",
    "    return [m.group(0) for m in camel_case_re.finditer(identifier)]\n",
    "\n",
    "def snake_case_split(identifier):\n",
    "    return identifier.split(\"_\")\n",
    "\n",
    "def split_method_name(method_name: str):\n",
    "    toks = [method_name]\n",
    "    toks = [tok for s in toks for tok in snake_case_split(s)]\n",
    "    toks = [tok.lower() for s in toks for tok in camel_case_split(s)]\n",
    "    return toks\n",
    "\n",
    "def filtered_row(pd_row):\n",
    "    out_funcs = pd_row['original_string']\n",
    "\n",
    "    # Fix cropped \"function\" token at the begging of the function string\n",
    "    for regex in _fix_function_crop_regexes:\n",
    "        out_funcs = regex.sub(r\"function\\1\", out_funcs, count=1)\n",
    "\n",
    "    # Remove function name from declaration, but leave it in the function body\n",
    "    _function_name_regex = r\"(function\\s*)\" + re.escape(pd_row['func_name'])\n",
    "    return re.sub(_function_name_regex, r\"\\1x\", out_funcs, count=1)\n",
    "\n",
    "# to create subtoken input as specified by paper\n",
    "def tok_and_pack_string(string):\n",
    "    out_toks = []\n",
    "    for tok in string.split():\n",
    "        out_toks.append(' '.join(split_method_name(tok)))\n",
    "    return ' '.join(out_toks)\n",
    "\n",
    "def process_data(jsonl_file):\n",
    "    data = []\n",
    "    with jsonlines.open(jsonl_file) as f:\n",
    "        for obj in f:\n",
    "            data.append(obj)\n",
    "    df = pd.DataFrame(data)\n",
    "    df['func_name'].replace('', np.nan, inplace=True)\n",
    "    df['original_string'].replace('', np.nan, inplace=True)\n",
    "    df.dropna(subset=['func_name', 'original_string'], inplace=True)\n",
    "    \n",
    "    df['func_name_pieces'] = df['func_name'].apply(split_method_name).apply(lambda x: ' '.join(x))\n",
    "    df['code_cleaned'] = df[['original_string', 'func_name']].progress_apply(filtered_row, axis=1).apply(lambda x: ' '.join(x.split()))\n",
    "    df['code_cleaned_subtoken'] = df['code_cleaned'].progress_apply(tok_and_pack_string)\n",
    "    \n",
    "    programs = df['code_cleaned_subtoken'].tolist()\n",
    "    programs_subtok = df['code_cleaned_subtoken'].tolist()\n",
    "    labels = df['func_name_pieces'].tolist()\n",
    "    \n",
    "    return programs, programs_subtok, labels, df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing data from train\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 81487/81487 [00:18<00:00, 4526.84it/s]\n",
      "100%|██████████| 81487/81487 [00:24<00:00, 3328.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing data from dev\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5501/5501 [00:00<00:00, 5600.81it/s]\n",
      "100%|██████████| 5501/5501 [00:02<00:00, 2460.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing data from test\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4441/4441 [00:00<00:00, 4700.30it/s]\n",
      "100%|██████████| 4441/4441 [00:01<00:00, 3502.33it/s]\n"
     ]
    }
   ],
   "source": [
    "prefix_in = 'data/codesearchnet_javascript/'\n",
    "prefix_out = 'data/neuralcodesum/'\n",
    "\n",
    "datasets = {\n",
    "    'train': 'javascript_train_supervised.jsonl',\n",
    "    'dev': 'javascript_valid_0.jsonl',\n",
    "    'test': 'javascript_test_0.jsonl',\n",
    "}\n",
    "\n",
    "for setname, fname in datasets.items():\n",
    "    print(\"Processing data from\", setname)\n",
    "    os.makedirs(os.path.join(prefix_out, setname), exist_ok=True)\n",
    "    programs, programs_subtok, labels, df = process_data(os.path.join(prefix_in, fname))\n",
    "    with open(os.path.join(prefix_out, setname, 'code.original'), 'w') as f:\n",
    "        f.write('\\n'.join(programs))\n",
    "    with open(os.path.join(prefix_out, setname, 'code.original_subtoken'), 'w') as f:\n",
    "        f.write('\\n'.join(programs_subtok))\n",
    "    with open(os.path.join(prefix_out, setname, 'javadoc.original'), 'w') as f:\n",
    "        f.write('\\n'.join(labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>repo</th>\n",
       "      <th>path</th>\n",
       "      <th>func_name</th>\n",
       "      <th>original_string</th>\n",
       "      <th>language</th>\n",
       "      <th>code</th>\n",
       "      <th>code_tokens</th>\n",
       "      <th>docstring</th>\n",
       "      <th>docstring_tokens</th>\n",
       "      <th>sha</th>\n",
       "      <th>url</th>\n",
       "      <th>partition</th>\n",
       "      <th>func_name_pieces</th>\n",
       "      <th>code_cleaned</th>\n",
       "      <th>code_cleaned_subtoken</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>axios/axios</td>\n",
       "      <td>lib/axios.js</td>\n",
       "      <td>createInstance</td>\n",
       "      <td>function createInstance(defaultConfig) {\\n  va...</td>\n",
       "      <td>javascript</td>\n",
       "      <td>function createInstance(defaultConfig) {\\n  va...</td>\n",
       "      <td>[function, createInstance, (, defaultConfig, )...</td>\n",
       "      <td>Create an instance of Axios\\n\\n@param {Object}...</td>\n",
       "      <td>[Create, an, instance, of, Axios]</td>\n",
       "      <td>92d231387fe2092f8736bc1746d4caa766b675f5</td>\n",
       "      <td>https://github.com/axios/axios/blob/92d231387f...</td>\n",
       "      <td>test</td>\n",
       "      <td>create instance</td>\n",
       "      <td>function x(defaultConfig) { var context = new ...</td>\n",
       "      <td>function x(default config) { var context = new...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>axios/axios</td>\n",
       "      <td>lib/cancel/CancelToken.js</td>\n",
       "      <td>CancelToken</td>\n",
       "      <td>function CancelToken(executor) {\\n  if (typeof...</td>\n",
       "      <td>javascript</td>\n",
       "      <td>function CancelToken(executor) {\\n  if (typeof...</td>\n",
       "      <td>[function, CancelToken, (, executor, ), {, if,...</td>\n",
       "      <td>A `CancelToken` is an object that can be used ...</td>\n",
       "      <td>[A, CancelToken, is, an, object, that, can, be...</td>\n",
       "      <td>92d231387fe2092f8736bc1746d4caa766b675f5</td>\n",
       "      <td>https://github.com/axios/axios/blob/92d231387f...</td>\n",
       "      <td>test</td>\n",
       "      <td>cancel token</td>\n",
       "      <td>function x(executor) { if (typeof executor !==...</td>\n",
       "      <td>function x(executor) { if (typeof executor !==...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>axios/axios</td>\n",
       "      <td>lib/utils.js</td>\n",
       "      <td>isArrayBufferView</td>\n",
       "      <td>function isArrayBufferView(val) {\\n  var resul...</td>\n",
       "      <td>javascript</td>\n",
       "      <td>function isArrayBufferView(val) {\\n  var resul...</td>\n",
       "      <td>[function, isArrayBufferView, (, val, ), {, va...</td>\n",
       "      <td>Determine if a value is a view on an ArrayBuff...</td>\n",
       "      <td>[Determine, if, a, value, is, a, view, on, an,...</td>\n",
       "      <td>92d231387fe2092f8736bc1746d4caa766b675f5</td>\n",
       "      <td>https://github.com/axios/axios/blob/92d231387f...</td>\n",
       "      <td>test</td>\n",
       "      <td>is array buffer view</td>\n",
       "      <td>function x(val) { var result; if ((typeof Arra...</td>\n",
       "      <td>function x(val) { var result; if ((typeof arra...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>axios/axios</td>\n",
       "      <td>lib/utils.js</td>\n",
       "      <td>isStandardBrowserEnv</td>\n",
       "      <td>function isStandardBrowserEnv() {\\n  if (typeo...</td>\n",
       "      <td>javascript</td>\n",
       "      <td>function isStandardBrowserEnv() {\\n  if (typeo...</td>\n",
       "      <td>[function, isStandardBrowserEnv, (, ), {, if, ...</td>\n",
       "      <td>Determine if we're running in a standard brows...</td>\n",
       "      <td>[Determine, if, we, re, running, in, a, standa...</td>\n",
       "      <td>92d231387fe2092f8736bc1746d4caa766b675f5</td>\n",
       "      <td>https://github.com/axios/axios/blob/92d231387f...</td>\n",
       "      <td>test</td>\n",
       "      <td>is standard browser env</td>\n",
       "      <td>function x() { if (typeof navigator !== 'undef...</td>\n",
       "      <td>function x() { if (typeof navigator !== 'undef...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>axios/axios</td>\n",
       "      <td>lib/utils.js</td>\n",
       "      <td>forEach</td>\n",
       "      <td>function forEach(obj, fn) {\\n  // Don't bother...</td>\n",
       "      <td>javascript</td>\n",
       "      <td>function forEach(obj, fn) {\\n  // Don't bother...</td>\n",
       "      <td>[function, forEach, (, obj, ,, fn, ), {, // Do...</td>\n",
       "      <td>Iterate over an Array or an Object invoking a ...</td>\n",
       "      <td>[Iterate, over, an, Array, or, an, Object, inv...</td>\n",
       "      <td>92d231387fe2092f8736bc1746d4caa766b675f5</td>\n",
       "      <td>https://github.com/axios/axios/blob/92d231387f...</td>\n",
       "      <td>test</td>\n",
       "      <td>for each</td>\n",
       "      <td>function x(obj, fn) { // Don't bother if no va...</td>\n",
       "      <td>function x(obj, fn) { // don't bother if no va...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6466</th>\n",
       "      <td>greatbsky/ES6MVC</td>\n",
       "      <td>src/main/Project.js</td>\n",
       "      <td>requireAllInterceptor</td>\n",
       "      <td>function requireAllInterceptor(dir) {\\n    if ...</td>\n",
       "      <td>javascript</td>\n",
       "      <td>function requireAllInterceptor(dir) {\\n    if ...</td>\n",
       "      <td>[function, requireAllInterceptor, (, dir, ), {...</td>\n",
       "      <td>/*自动require interceptors</td>\n",
       "      <td>[/, *, 自动require, interceptors]</td>\n",
       "      <td>b28ebd9931a0228e2ef098ff6578b3b0607a4da8</td>\n",
       "      <td>https://github.com/greatbsky/ES6MVC/blob/b28eb...</td>\n",
       "      <td>test</td>\n",
       "      <td>require all interceptor</td>\n",
       "      <td>function x(dir) { if (!fs.existsSync(dir)) { r...</td>\n",
       "      <td>function x(dir) { if (!fs.exists sync(dir)) { ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6476</th>\n",
       "      <td>tuddman/passport-wink</td>\n",
       "      <td>lib/passport-wink/strategy.js</td>\n",
       "      <td>Strategy</td>\n",
       "      <td>function Strategy(options, verify) {\\n  option...</td>\n",
       "      <td>javascript</td>\n",
       "      <td>function Strategy(options, verify) {\\n  option...</td>\n",
       "      <td>[function, Strategy, (, options, ,, verify, ),...</td>\n",
       "      <td>`Strategy` constructor.\\n\\nThe wink authentica...</td>\n",
       "      <td>[Strategy, constructor, .]</td>\n",
       "      <td>010d2a843af645dad931a414605029c81649d59c</td>\n",
       "      <td>https://github.com/tuddman/passport-wink/blob/...</td>\n",
       "      <td>test</td>\n",
       "      <td>strategy</td>\n",
       "      <td>function x(options, verify) { options = option...</td>\n",
       "      <td>function x(options, verify) { options = option...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6477</th>\n",
       "      <td>farmdawgnation/vain</td>\n",
       "      <td>lib/vain.js</td>\n",
       "      <td>processParams</td>\n",
       "      <td>function processParams(paramsString) {\\n  var ...</td>\n",
       "      <td>javascript</td>\n",
       "      <td>function processParams(paramsString) {\\n  var ...</td>\n",
       "      <td>[function, processParams, (, paramsString, ), ...</td>\n",
       "      <td>Utility method to process a params string in a...</td>\n",
       "      <td>[Utility, method, to, process, a, params, stri...</td>\n",
       "      <td>0a30de09f443c9c313223b338a8892b6ef9ddf1d</td>\n",
       "      <td>https://github.com/farmdawgnation/vain/blob/0a...</td>\n",
       "      <td>test</td>\n",
       "      <td>process params</td>\n",
       "      <td>function x(paramsString) { var individualParam...</td>\n",
       "      <td>function x(params string) { var individual par...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6480</th>\n",
       "      <td>beyo/model</td>\n",
       "      <td>lib/collection.js</td>\n",
       "      <td>Collection</td>\n",
       "      <td>function Collection(options) {\\n  if (!(this i...</td>\n",
       "      <td>javascript</td>\n",
       "      <td>function Collection(options) {\\n  if (!(this i...</td>\n",
       "      <td>[function, Collection, (, options, ), {, if, (...</td>\n",
       "      <td>Create a new collection\\n\\n@param {Object} opt...</td>\n",
       "      <td>[Create, a, new, collection]</td>\n",
       "      <td>625b386b6e1141e1869ede753640850c0c9f0971</td>\n",
       "      <td>https://github.com/beyo/model/blob/625b386b6e1...</td>\n",
       "      <td>test</td>\n",
       "      <td>collection</td>\n",
       "      <td>function x(options) { if (!(this instanceof Co...</td>\n",
       "      <td>function x(options) { if (!(this instanceof co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6481</th>\n",
       "      <td>beyo/model</td>\n",
       "      <td>lib/collection.js</td>\n",
       "      <td>find</td>\n",
       "      <td>function find(filter) {\\n  var item;\\n  var i;...</td>\n",
       "      <td>javascript</td>\n",
       "      <td>function find(filter) {\\n  var item;\\n  var i;...</td>\n",
       "      <td>[function, find, (, filter, ), {, var, item, ;...</td>\n",
       "      <td>Find the first item matching filter\\n\\n@param ...</td>\n",
       "      <td>[Find, the, first, item, matching, filter]</td>\n",
       "      <td>625b386b6e1141e1869ede753640850c0c9f0971</td>\n",
       "      <td>https://github.com/beyo/model/blob/625b386b6e1...</td>\n",
       "      <td>test</td>\n",
       "      <td>find</td>\n",
       "      <td>function x(filter) { var item; var i; var ilen...</td>\n",
       "      <td>function x(filter) { var item; var i; var ilen...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4441 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       repo                           path  \\\n",
       "0               axios/axios                   lib/axios.js   \n",
       "1               axios/axios      lib/cancel/CancelToken.js   \n",
       "2               axios/axios                   lib/utils.js   \n",
       "3               axios/axios                   lib/utils.js   \n",
       "4               axios/axios                   lib/utils.js   \n",
       "...                     ...                            ...   \n",
       "6466       greatbsky/ES6MVC            src/main/Project.js   \n",
       "6476  tuddman/passport-wink  lib/passport-wink/strategy.js   \n",
       "6477    farmdawgnation/vain                    lib/vain.js   \n",
       "6480             beyo/model              lib/collection.js   \n",
       "6481             beyo/model              lib/collection.js   \n",
       "\n",
       "                  func_name  \\\n",
       "0            createInstance   \n",
       "1               CancelToken   \n",
       "2         isArrayBufferView   \n",
       "3      isStandardBrowserEnv   \n",
       "4                   forEach   \n",
       "...                     ...   \n",
       "6466  requireAllInterceptor   \n",
       "6476               Strategy   \n",
       "6477          processParams   \n",
       "6480             Collection   \n",
       "6481                   find   \n",
       "\n",
       "                                        original_string    language  \\\n",
       "0     function createInstance(defaultConfig) {\\n  va...  javascript   \n",
       "1     function CancelToken(executor) {\\n  if (typeof...  javascript   \n",
       "2     function isArrayBufferView(val) {\\n  var resul...  javascript   \n",
       "3     function isStandardBrowserEnv() {\\n  if (typeo...  javascript   \n",
       "4     function forEach(obj, fn) {\\n  // Don't bother...  javascript   \n",
       "...                                                 ...         ...   \n",
       "6466  function requireAllInterceptor(dir) {\\n    if ...  javascript   \n",
       "6476  function Strategy(options, verify) {\\n  option...  javascript   \n",
       "6477  function processParams(paramsString) {\\n  var ...  javascript   \n",
       "6480  function Collection(options) {\\n  if (!(this i...  javascript   \n",
       "6481  function find(filter) {\\n  var item;\\n  var i;...  javascript   \n",
       "\n",
       "                                                   code  \\\n",
       "0     function createInstance(defaultConfig) {\\n  va...   \n",
       "1     function CancelToken(executor) {\\n  if (typeof...   \n",
       "2     function isArrayBufferView(val) {\\n  var resul...   \n",
       "3     function isStandardBrowserEnv() {\\n  if (typeo...   \n",
       "4     function forEach(obj, fn) {\\n  // Don't bother...   \n",
       "...                                                 ...   \n",
       "6466  function requireAllInterceptor(dir) {\\n    if ...   \n",
       "6476  function Strategy(options, verify) {\\n  option...   \n",
       "6477  function processParams(paramsString) {\\n  var ...   \n",
       "6480  function Collection(options) {\\n  if (!(this i...   \n",
       "6481  function find(filter) {\\n  var item;\\n  var i;...   \n",
       "\n",
       "                                            code_tokens  \\\n",
       "0     [function, createInstance, (, defaultConfig, )...   \n",
       "1     [function, CancelToken, (, executor, ), {, if,...   \n",
       "2     [function, isArrayBufferView, (, val, ), {, va...   \n",
       "3     [function, isStandardBrowserEnv, (, ), {, if, ...   \n",
       "4     [function, forEach, (, obj, ,, fn, ), {, // Do...   \n",
       "...                                                 ...   \n",
       "6466  [function, requireAllInterceptor, (, dir, ), {...   \n",
       "6476  [function, Strategy, (, options, ,, verify, ),...   \n",
       "6477  [function, processParams, (, paramsString, ), ...   \n",
       "6480  [function, Collection, (, options, ), {, if, (...   \n",
       "6481  [function, find, (, filter, ), {, var, item, ;...   \n",
       "\n",
       "                                              docstring  \\\n",
       "0     Create an instance of Axios\\n\\n@param {Object}...   \n",
       "1     A `CancelToken` is an object that can be used ...   \n",
       "2     Determine if a value is a view on an ArrayBuff...   \n",
       "3     Determine if we're running in a standard brows...   \n",
       "4     Iterate over an Array or an Object invoking a ...   \n",
       "...                                                 ...   \n",
       "6466                           /*自动require interceptors   \n",
       "6476  `Strategy` constructor.\\n\\nThe wink authentica...   \n",
       "6477  Utility method to process a params string in a...   \n",
       "6480  Create a new collection\\n\\n@param {Object} opt...   \n",
       "6481  Find the first item matching filter\\n\\n@param ...   \n",
       "\n",
       "                                       docstring_tokens  \\\n",
       "0                     [Create, an, instance, of, Axios]   \n",
       "1     [A, CancelToken, is, an, object, that, can, be...   \n",
       "2     [Determine, if, a, value, is, a, view, on, an,...   \n",
       "3     [Determine, if, we, re, running, in, a, standa...   \n",
       "4     [Iterate, over, an, Array, or, an, Object, inv...   \n",
       "...                                                 ...   \n",
       "6466                    [/, *, 自动require, interceptors]   \n",
       "6476                         [Strategy, constructor, .]   \n",
       "6477  [Utility, method, to, process, a, params, stri...   \n",
       "6480                       [Create, a, new, collection]   \n",
       "6481         [Find, the, first, item, matching, filter]   \n",
       "\n",
       "                                           sha  \\\n",
       "0     92d231387fe2092f8736bc1746d4caa766b675f5   \n",
       "1     92d231387fe2092f8736bc1746d4caa766b675f5   \n",
       "2     92d231387fe2092f8736bc1746d4caa766b675f5   \n",
       "3     92d231387fe2092f8736bc1746d4caa766b675f5   \n",
       "4     92d231387fe2092f8736bc1746d4caa766b675f5   \n",
       "...                                        ...   \n",
       "6466  b28ebd9931a0228e2ef098ff6578b3b0607a4da8   \n",
       "6476  010d2a843af645dad931a414605029c81649d59c   \n",
       "6477  0a30de09f443c9c313223b338a8892b6ef9ddf1d   \n",
       "6480  625b386b6e1141e1869ede753640850c0c9f0971   \n",
       "6481  625b386b6e1141e1869ede753640850c0c9f0971   \n",
       "\n",
       "                                                    url partition  \\\n",
       "0     https://github.com/axios/axios/blob/92d231387f...      test   \n",
       "1     https://github.com/axios/axios/blob/92d231387f...      test   \n",
       "2     https://github.com/axios/axios/blob/92d231387f...      test   \n",
       "3     https://github.com/axios/axios/blob/92d231387f...      test   \n",
       "4     https://github.com/axios/axios/blob/92d231387f...      test   \n",
       "...                                                 ...       ...   \n",
       "6466  https://github.com/greatbsky/ES6MVC/blob/b28eb...      test   \n",
       "6476  https://github.com/tuddman/passport-wink/blob/...      test   \n",
       "6477  https://github.com/farmdawgnation/vain/blob/0a...      test   \n",
       "6480  https://github.com/beyo/model/blob/625b386b6e1...      test   \n",
       "6481  https://github.com/beyo/model/blob/625b386b6e1...      test   \n",
       "\n",
       "             func_name_pieces  \\\n",
       "0             create instance   \n",
       "1                cancel token   \n",
       "2        is array buffer view   \n",
       "3     is standard browser env   \n",
       "4                    for each   \n",
       "...                       ...   \n",
       "6466  require all interceptor   \n",
       "6476                 strategy   \n",
       "6477           process params   \n",
       "6480               collection   \n",
       "6481                     find   \n",
       "\n",
       "                                           code_cleaned  \\\n",
       "0     function x(defaultConfig) { var context = new ...   \n",
       "1     function x(executor) { if (typeof executor !==...   \n",
       "2     function x(val) { var result; if ((typeof Arra...   \n",
       "3     function x() { if (typeof navigator !== 'undef...   \n",
       "4     function x(obj, fn) { // Don't bother if no va...   \n",
       "...                                                 ...   \n",
       "6466  function x(dir) { if (!fs.existsSync(dir)) { r...   \n",
       "6476  function x(options, verify) { options = option...   \n",
       "6477  function x(paramsString) { var individualParam...   \n",
       "6480  function x(options) { if (!(this instanceof Co...   \n",
       "6481  function x(filter) { var item; var i; var ilen...   \n",
       "\n",
       "                                  code_cleaned_subtoken  \n",
       "0     function x(default config) { var context = new...  \n",
       "1     function x(executor) { if (typeof executor !==...  \n",
       "2     function x(val) { var result; if ((typeof arra...  \n",
       "3     function x() { if (typeof navigator !== 'undef...  \n",
       "4     function x(obj, fn) { // don't bother if no va...  \n",
       "...                                                 ...  \n",
       "6466  function x(dir) { if (!fs.exists sync(dir)) { ...  \n",
       "6476  function x(options, verify) { options = option...  \n",
       "6477  function x(params string) { var individual par...  \n",
       "6480  function x(options) { if (!(this instanceof co...  \n",
       "6481  function x(filter) { var item; var i; var ilen...  \n",
       "\n",
       "[4441 rows x 15 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
